from lxml import etree
import re
from proxy_helper import ProxyHelper


class LianjiaZufangSpider():

    def __init__(self):
        self.total = 0
        self.proxy_helper = ProxyHelper()

    def run(self):
        url = f"https://zz.lianjia.com/zufang/pg1/#contentList"
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
        }
        response = self.proxy_helper.requests_with_proxy(url, headers=headers)
        result = response.content.decode()
        root = etree.HTML(result)
        # 获取城市的所有区
        area_list = root.xpath("//div[@id='filter']/ul[2]/li[@class='filter__item--level2  ']/a")
        for area in area_list:
            area_name = area.xpath("./text()")[0]
            area_url = "https://zz.lianjia.com" + area.xpath("./@href")[0]
            # 计算不同区的租房数量
            response = self.proxy_helper.requests_with_proxy(area_url, headers=headers)
            html_str = response.content.decode()
            area_total = re.search('<span class="content__title--hl">(\d+)</span>', html_str).group(1)
            area_total = int(area_total)
            if area_total > 3000:
                # 计算区域下一级商圈的租房数量
                root = etree.HTML(html_str)
                shopping_list = root.xpath("//div[@id='filter']/ul[4]/li[@class='filter__item--level3  ']/a")
                for shopping in shopping_list:
                    shopping_name = shopping.xpath("./text()")[0]
                    shopping_url = "https://zz.lianjia.com" + shopping.xpath("./@href")[0]
                    # 计算商圈下的租房数量
                    response = self.proxy_helper.requests_with_proxy(shopping_url, headers=headers)
                    html_str = response.content.decode()
                    shopping_total = re.search('<span class="content__title--hl">(\d+)</span>', html_str).group(1)
                    shopping_total = int(shopping_total)
                    self.total += shopping_total
                    print(area_name, shopping_name, shopping_total, self.total)
            else:
                self.total += area_total
                print(area_name, area_total, self.total)


if __name__ == '__main__':
    LianjiaZufangSpider().run()
